{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: 自定义nn模块\n",
    "--------------------------\n",
    "\n",
    "对于复杂的网络结构，我们可以通过基础Module了自定义nn模块。这样的好处是用一个类来同样管理，而且\n",
    "更容易复用代码。\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 686.2594604492188\n",
      "1 631.3014526367188\n",
      "2 584.796142578125\n",
      "3 544.9374389648438\n",
      "4 510.06353759765625\n",
      "5 479.10888671875\n",
      "6 451.3166809082031\n",
      "7 426.2210693359375\n",
      "8 403.3813781738281\n",
      "9 382.250244140625\n",
      "10 362.5904541015625\n",
      "11 344.0471496582031\n",
      "12 326.5711975097656\n",
      "13 310.12109375\n",
      "14 294.5249328613281\n",
      "15 279.757568359375\n",
      "16 265.7481994628906\n",
      "17 252.41299438476562\n",
      "18 239.6514434814453\n",
      "19 227.4420623779297\n",
      "20 215.7503662109375\n",
      "21 204.62005615234375\n",
      "22 193.9403533935547\n",
      "23 183.7403564453125\n",
      "24 173.9613800048828\n",
      "25 164.56796264648438\n",
      "26 155.60276794433594\n",
      "27 147.0603485107422\n",
      "28 138.92303466796875\n",
      "29 131.1431427001953\n",
      "30 123.73912811279297\n",
      "31 116.6927490234375\n",
      "32 109.99571228027344\n",
      "33 103.63246154785156\n",
      "34 97.60087585449219\n",
      "35 91.88713073730469\n",
      "36 86.48358154296875\n",
      "37 81.36537170410156\n",
      "38 76.52302551269531\n",
      "39 71.9395980834961\n",
      "40 67.61711883544922\n",
      "41 63.53286361694336\n",
      "42 59.681617736816406\n",
      "43 56.04635238647461\n",
      "44 52.625831604003906\n",
      "45 49.408058166503906\n",
      "46 46.38482666015625\n",
      "47 43.54777145385742\n",
      "48 40.884220123291016\n",
      "49 38.38938522338867\n",
      "50 36.0556526184082\n",
      "51 33.86988067626953\n",
      "52 31.8211612701416\n",
      "53 29.90234375\n",
      "54 28.10257911682129\n",
      "55 26.418472290039062\n",
      "56 24.8418025970459\n",
      "57 23.365432739257812\n",
      "58 21.98244285583496\n",
      "59 20.686222076416016\n",
      "60 19.47402000427246\n",
      "61 18.338350296020508\n",
      "62 17.273841857910156\n",
      "63 16.27958106994629\n",
      "64 15.34726333618164\n",
      "65 14.474173545837402\n",
      "66 13.656461715698242\n",
      "67 12.889588356018066\n",
      "68 12.169529914855957\n",
      "69 11.493806838989258\n",
      "70 10.86031723022461\n",
      "71 10.263574600219727\n",
      "72 9.702670097351074\n",
      "73 9.176104545593262\n",
      "74 8.681727409362793\n",
      "75 8.217498779296875\n",
      "76 7.78070068359375\n",
      "77 7.369674205780029\n",
      "78 6.982870101928711\n",
      "79 6.619162082672119\n",
      "80 6.276303768157959\n",
      "81 5.953715801239014\n",
      "82 5.64962911605835\n",
      "83 5.363315582275391\n",
      "84 5.0929155349731445\n",
      "85 4.838088035583496\n",
      "86 4.597591400146484\n",
      "87 4.370265007019043\n",
      "88 4.1555256843566895\n",
      "89 3.9525794982910156\n",
      "90 3.760725498199463\n",
      "91 3.5794613361358643\n",
      "92 3.4077768325805664\n",
      "93 3.2452759742736816\n",
      "94 3.091123580932617\n",
      "95 2.9451494216918945\n",
      "96 2.806915044784546\n",
      "97 2.67592453956604\n",
      "98 2.5517592430114746\n",
      "99 2.434028387069702\n",
      "100 2.3223745822906494\n",
      "101 2.216444969177246\n",
      "102 2.115831136703491\n",
      "103 2.020437002182007\n",
      "104 1.9298652410507202\n",
      "105 1.8437539339065552\n",
      "106 1.7617982625961304\n",
      "107 1.6838340759277344\n",
      "108 1.6097533702850342\n",
      "109 1.5392441749572754\n",
      "110 1.472186803817749\n",
      "111 1.4082980155944824\n",
      "112 1.3474677801132202\n",
      "113 1.2894623279571533\n",
      "114 1.2341868877410889\n",
      "115 1.1815515756607056\n",
      "116 1.131364345550537\n",
      "117 1.0835613012313843\n",
      "118 1.0379600524902344\n",
      "119 0.9944199323654175\n",
      "120 0.9528597593307495\n",
      "121 0.9132634401321411\n",
      "122 0.8756293654441833\n",
      "123 0.8397300243377686\n",
      "124 0.805433988571167\n",
      "125 0.7726611495018005\n",
      "126 0.7413666248321533\n",
      "127 0.7114477157592773\n",
      "128 0.6828161478042603\n",
      "129 0.6554570198059082\n",
      "130 0.6292985081672668\n",
      "131 0.604280948638916\n",
      "132 0.5803555250167847\n",
      "133 0.5574570298194885\n",
      "134 0.5355619192123413\n",
      "135 0.5146176218986511\n",
      "136 0.4945870339870453\n",
      "137 0.4753904640674591\n",
      "138 0.4570006728172302\n",
      "139 0.4394027292728424\n",
      "140 0.42254146933555603\n",
      "141 0.40637877583503723\n",
      "142 0.390934020280838\n",
      "143 0.37611842155456543\n",
      "144 0.3619283437728882\n",
      "145 0.34830722212791443\n",
      "146 0.33524975180625916\n",
      "147 0.3227214515209198\n",
      "148 0.31069013476371765\n",
      "149 0.29915061593055725\n",
      "150 0.28807488083839417\n",
      "151 0.2774510979652405\n",
      "152 0.2672513425350189\n",
      "153 0.2574580907821655\n",
      "154 0.24804610013961792\n",
      "155 0.23877954483032227\n",
      "156 0.22988823056221008\n",
      "157 0.22136564552783966\n",
      "158 0.21318143606185913\n",
      "159 0.20533186197280884\n",
      "160 0.19780901074409485\n",
      "161 0.1905868649482727\n",
      "162 0.18365328013896942\n",
      "163 0.17699673771858215\n",
      "164 0.17059734463691711\n",
      "165 0.16444864869117737\n",
      "166 0.15853972733020782\n",
      "167 0.15285737812519073\n",
      "168 0.14740589261054993\n",
      "169 0.14216160774230957\n",
      "170 0.13711676001548767\n",
      "171 0.13226358592510223\n",
      "172 0.12759670615196228\n",
      "173 0.12310992926359177\n",
      "174 0.11879114806652069\n",
      "175 0.11463826894760132\n",
      "176 0.11064040660858154\n",
      "177 0.10679415613412857\n",
      "178 0.10309237241744995\n",
      "179 0.0995292142033577\n",
      "180 0.09609568864107132\n",
      "181 0.0927930474281311\n",
      "182 0.08961179107427597\n",
      "183 0.08654838055372238\n",
      "184 0.08359683305025101\n",
      "185 0.08075377345085144\n",
      "186 0.07801400870084763\n",
      "187 0.07537535578012466\n",
      "188 0.07283208519220352\n",
      "189 0.07037939876317978\n",
      "190 0.06801652908325195\n",
      "191 0.06573743373155594\n",
      "192 0.06354115158319473\n",
      "193 0.06142434850335121\n",
      "194 0.05938328057527542\n",
      "195 0.05741359293460846\n",
      "196 0.05551448091864586\n",
      "197 0.05368218943476677\n",
      "198 0.05191481113433838\n",
      "199 0.05021132156252861\n",
      "200 0.04856682941317558\n",
      "201 0.04697948694229126\n",
      "202 0.045448917895555496\n",
      "203 0.043970149010419846\n",
      "204 0.04254346713423729\n",
      "205 0.04116659611463547\n",
      "206 0.039837583899497986\n",
      "207 0.03855311498045921\n",
      "208 0.03731384128332138\n",
      "209 0.036116812378168106\n",
      "210 0.03496103733778\n",
      "211 0.03384438529610634\n",
      "212 0.03276515379548073\n",
      "213 0.031722649931907654\n",
      "214 0.030716171488165855\n",
      "215 0.029744260013103485\n",
      "216 0.02880498208105564\n",
      "217 0.027897058054804802\n",
      "218 0.02701951377093792\n",
      "219 0.026171525940299034\n",
      "220 0.025352047756314278\n",
      "221 0.02455979585647583\n",
      "222 0.023794101551175117\n",
      "223 0.02305399812757969\n",
      "224 0.022337937727570534\n",
      "225 0.02164526656270027\n",
      "226 0.02097555808722973\n",
      "227 0.020327642560005188\n",
      "228 0.019701115787029266\n",
      "229 0.01909530907869339\n",
      "230 0.018508991226553917\n",
      "231 0.017941588535904884\n",
      "232 0.017392707988619804\n",
      "233 0.01686140149831772\n",
      "234 0.016347207129001617\n",
      "235 0.015849918127059937\n",
      "236 0.015368361957371235\n",
      "237 0.014902160502970219\n",
      "238 0.014451056718826294\n",
      "239 0.014014440588653088\n",
      "240 0.013591606169939041\n",
      "241 0.013182281516492367\n",
      "242 0.012785986065864563\n",
      "243 0.012402184307575226\n",
      "244 0.01203029602766037\n",
      "245 0.011670324951410294\n",
      "246 0.011321629397571087\n",
      "247 0.010984260588884354\n",
      "248 0.010657208040356636\n",
      "249 0.010340196080505848\n",
      "250 0.010033304803073406\n",
      "251 0.00973595678806305\n",
      "252 0.009447920136153698\n",
      "253 0.009168713353574276\n",
      "254 0.008898149244487286\n",
      "255 0.008636103011667728\n",
      "256 0.008382044732570648\n",
      "257 0.008135756477713585\n",
      "258 0.007897119037806988\n",
      "259 0.007665828336030245\n",
      "260 0.007441507186740637\n",
      "261 0.007224107161164284\n",
      "262 0.007017266470938921\n",
      "263 0.006817416753619909\n",
      "264 0.006623750552535057\n",
      "265 0.006435899529606104\n",
      "266 0.0062538934871554375\n",
      "267 0.006077279336750507\n",
      "268 0.0059060826897621155\n",
      "269 0.00574014475569129\n",
      "270 0.005579248536378145\n",
      "271 0.005423067137598991\n",
      "272 0.005271578207612038\n",
      "273 0.005124684423208237\n",
      "274 0.0049821315333247185\n",
      "275 0.004843832924962044\n",
      "276 0.004709657747298479\n",
      "277 0.004579509142786264\n",
      "278 0.004453164990991354\n",
      "279 0.004330581985414028\n",
      "280 0.004211557563394308\n",
      "281 0.004096029791980982\n",
      "282 0.003983878996223211\n",
      "283 0.003875087248161435\n",
      "284 0.003769456408917904\n",
      "285 0.0036668917164206505\n",
      "286 0.003567283507436514\n",
      "287 0.0034706201404333115\n",
      "288 0.0033767365384846926\n",
      "289 0.0032855228055268526\n",
      "290 0.003196936333552003\n",
      "291 0.00311092846095562\n",
      "292 0.003027426777407527\n",
      "293 0.002946281572803855\n",
      "294 0.002867457689717412\n",
      "295 0.002790878526866436\n",
      "296 0.0027164716739207506\n",
      "297 0.002644247142598033\n",
      "298 0.002574007259681821\n",
      "299 0.0025058020837605\n",
      "300 0.0024394949432462454\n",
      "301 0.0023750667460262775\n",
      "302 0.002312423661351204\n",
      "303 0.002251541009172797\n",
      "304 0.0021923789754509926\n",
      "305 0.002134919399395585\n",
      "306 0.002078989753499627\n",
      "307 0.002024643123149872\n",
      "308 0.00197183177806437\n",
      "309 0.001920489827170968\n",
      "310 0.0018705540569499135\n",
      "311 0.0018219854682683945\n",
      "312 0.001774778706021607\n",
      "313 0.0017288774251937866\n",
      "314 0.0016842280747368932\n",
      "315 0.00164079072419554\n",
      "316 0.0015985488425940275\n",
      "317 0.0015574877616018057\n",
      "318 0.0015175191219896078\n",
      "319 0.0014786531683057547\n",
      "320 0.0014408212155103683\n",
      "321 0.0014040363021194935\n",
      "322 0.001368258846923709\n",
      "323 0.0013334389077499509\n",
      "324 0.0012995402794331312\n",
      "325 0.0012665895046666265\n",
      "326 0.0012344788992777467\n",
      "327 0.0012032696977257729\n",
      "328 0.001172887859866023\n",
      "329 0.0011433204635977745\n",
      "330 0.001114533399231732\n",
      "331 0.0010865204967558384\n",
      "332 0.0010592498583719134\n",
      "333 0.0010327105410397053\n",
      "334 0.0010068565607070923\n",
      "335 0.0009817015379667282\n",
      "336 0.000957207172177732\n",
      "337 0.0009333763737231493\n",
      "338 0.0009101526811718941\n",
      "339 0.0008875550702214241\n",
      "340 0.0008655337733216584\n",
      "341 0.0008440891397185624\n",
      "342 0.0008232113323174417\n",
      "343 0.0008028754382394254\n",
      "344 0.0007830737740732729\n",
      "345 0.0007637906819581985\n",
      "346 0.0007449908298440278\n",
      "347 0.0007267004693858325\n",
      "348 0.0007088782149367034\n",
      "349 0.0006915233097970486\n",
      "350 0.0006745982682332397\n",
      "351 0.0006581273046322167\n",
      "352 0.0006420681020244956\n",
      "353 0.0006264123949222267\n",
      "354 0.0006111772381700575\n",
      "355 0.0005963261355645955\n",
      "356 0.0005818486679345369\n",
      "357 0.0005677472800016403\n",
      "358 0.0005540119600482285\n",
      "359 0.0005406126147136092\n",
      "360 0.0005275557050481439\n",
      "361 0.0005148384370841086\n",
      "362 0.0005024460842832923\n",
      "363 0.0004903509397991002\n",
      "364 0.00047857200843282044\n",
      "365 0.00046708827721886337\n",
      "366 0.0004558963992167264\n",
      "367 0.0004449841217137873\n",
      "368 0.0004343533073551953\n",
      "369 0.0004239700792822987\n",
      "370 0.0004138603981118649\n",
      "371 0.00040400109719485044\n",
      "372 0.0003943882475141436\n",
      "373 0.0003850197826977819\n",
      "374 0.00037587148835882545\n",
      "375 0.0003669524157885462\n",
      "376 0.00035826745443046093\n",
      "377 0.00034979014890268445\n",
      "378 0.0003415268729440868\n",
      "379 0.0003334645298309624\n",
      "380 0.0003255998599343002\n",
      "381 0.0003179280902259052\n",
      "382 0.0003104456700384617\n",
      "383 0.00030315289041027427\n",
      "384 0.0002960303390864283\n",
      "385 0.00028909905813634396\n",
      "386 0.00028232031036168337\n",
      "387 0.00027571816463023424\n",
      "388 0.0002692755952011794\n",
      "389 0.0002629863447509706\n",
      "390 0.0002568456984590739\n",
      "391 0.0002508577599655837\n",
      "392 0.00024501807638444006\n",
      "393 0.00023932290787342936\n",
      "394 0.0002337596088182181\n",
      "395 0.00022832973627373576\n",
      "396 0.00022303094738163054\n",
      "397 0.0002178671129513532\n",
      "398 0.00021282557281665504\n",
      "399 0.00020790565758943558\n",
      "400 0.0002031034091487527\n",
      "401 0.00019841697940137237\n",
      "402 0.000193838044651784\n",
      "403 0.00018937527784146369\n",
      "404 0.00018501657177694142\n",
      "405 0.0001807632652344182\n",
      "406 0.0001766102504916489\n",
      "407 0.00017255698912777007\n",
      "408 0.00016859867901075631\n",
      "409 0.00016473566938657314\n",
      "410 0.0001609677419764921\n",
      "411 0.0001572836481500417\n",
      "412 0.00015368926688097417\n",
      "413 0.0001501814986113459\n",
      "414 0.00014675328566227108\n",
      "415 0.00014340836787596345\n",
      "416 0.00014014301996212453\n",
      "417 0.00013695504458155483\n",
      "418 0.00013384238991420716\n",
      "419 0.00013080096687190235\n",
      "420 0.00012782865087501705\n",
      "421 0.0001249331544386223\n",
      "422 0.00012209906708449125\n",
      "423 0.00011933469795621932\n",
      "424 0.00011663644545478746\n",
      "425 0.0001139963060268201\n",
      "426 0.00011142322182422504\n",
      "427 0.00010890879639191553\n",
      "428 0.00010645158909028396\n",
      "429 0.00010405346984043717\n",
      "430 0.00010171005851589143\n",
      "431 9.942201722878963e-05\n",
      "432 9.718444925965741e-05\n",
      "433 9.500460146227852e-05\n",
      "434 9.286970453104004e-05\n",
      "435 9.078363655135036e-05\n",
      "436 8.874951890902594e-05\n",
      "437 8.675820572534576e-05\n",
      "438 8.481980330543593e-05\n",
      "439 8.291965787066147e-05\n",
      "440 8.106514724204317e-05\n",
      "441 7.925427780719474e-05\n",
      "442 7.748592179268599e-05\n",
      "443 7.575883500976488e-05\n",
      "444 7.406806980725378e-05\n",
      "445 7.241530693136156e-05\n",
      "446 7.080676732584834e-05\n",
      "447 6.923100590938702e-05\n",
      "448 6.769159517716616e-05\n",
      "449 6.618585030082613e-05\n",
      "450 6.471577944466844e-05\n",
      "451 6.32814917480573e-05\n",
      "452 6.187564576976001e-05\n",
      "453 6.050529918866232e-05\n",
      "454 5.9164427511859685e-05\n",
      "455 5.785573011962697e-05\n",
      "456 5.6574026530142874e-05\n",
      "457 5.532176510314457e-05\n",
      "458 5.410234734881669e-05\n",
      "459 5.290694025461562e-05\n",
      "460 5.174061152501963e-05\n",
      "461 5.0598133384482935e-05\n",
      "462 4.948048808728345e-05\n",
      "463 4.839195389649831e-05\n",
      "464 4.7326841013273224e-05\n",
      "465 4.6283250412670895e-05\n",
      "466 4.526607153820805e-05\n",
      "467 4.427174644661136e-05\n",
      "468 4.329998046159744e-05\n",
      "469 4.234864900354296e-05\n",
      "470 4.1417682950850576e-05\n",
      "471 4.050982170156203e-05\n",
      "472 3.9622693293495104e-05\n",
      "473 3.8753052649553865e-05\n",
      "474 3.7905068893451244e-05\n",
      "475 3.707679934450425e-05\n",
      "476 3.6266075767343864e-05\n",
      "477 3.547223604982719e-05\n",
      "478 3.4695131034823135e-05\n",
      "479 3.393687438801862e-05\n",
      "480 3.319690586067736e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "481 3.247150016250089e-05\n",
      "482 3.176438985974528e-05\n",
      "483 3.107231532339938e-05\n",
      "484 3.039381954295095e-05\n",
      "485 2.973149821627885e-05\n",
      "486 2.90845400741091e-05\n",
      "487 2.845118251570966e-05\n",
      "488 2.7833642889163457e-05\n",
      "489 2.7228939870838076e-05\n",
      "490 2.6637020710040815e-05\n",
      "491 2.6059049559989944e-05\n",
      "492 2.549384589656256e-05\n",
      "493 2.494076943548862e-05\n",
      "494 2.439930358377751e-05\n",
      "495 2.386920459684916e-05\n",
      "496 2.3351785785052925e-05\n",
      "497 2.2845244529889897e-05\n",
      "498 2.2351927327690646e-05\n",
      "499 2.1868860130780376e-05\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class TwoLayerNet(torch.nn.Module):\n",
    "    def __init__(self, D_in, H, D_out):\n",
    "        \"\"\"\n",
    "        在构造函数里，我们定义两个nn.Linear模块，把它们保存到self里。\n",
    "        \"\"\"\n",
    "        super(TwoLayerNet, self).__init__()\n",
    "        self.linear1 = torch.nn.Linear(D_in, H)\n",
    "        self.linear2 = torch.nn.Linear(H, D_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        在forward函数里，我们需要根据网络结构来实现前向计算。通常我们会上定义的模块来计算。\n",
    "        \"\"\"\n",
    "        h_relu = self.linear1(x).clamp(min=0)\n",
    "        y_pred = self.linear2(h_relu)\n",
    "        return y_pred\n",
    "\n",
    " \n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    " \n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    " \n",
    "model = TwoLayerNet(D_in, H, D_out)\n",
    " \n",
    "criterion = torch.nn.MSELoss(reduction='sum')\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-4)\n",
    "for t in range(500): \n",
    "    y_pred = model(x)\n",
    " \n",
    "    loss = criterion(y_pred, y)\n",
    "    print(t, loss.item())\n",
    " \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.6-env",
   "language": "python",
   "name": "py3.6-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
